package fetch

import (
	"bufio"
	"fmt"
	log "github.com/sirupsen/logrus"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"golang.org/x/text/transform"
	"io/ioutil"
	"net/http"
	"time"
)

var rateLimiter = time.Tick(50 * time.Millisecond)
var timeout = time.Tick(30 * time.Minute)

func Fetch(url string) ([]byte, error) {

	//执行50ms之后，放入通道丢弃
	<-rateLimiter

	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36")
	resp, err := client.Do(req)
	if err != nil {
		log.Fatalln(err)
	}

	defer func() {
		err = resp.Body.Close()
		if err != nil {
			log.Warnf("response close err , ignore", err.Error())
		}
	}()

	if resp.StatusCode != http.StatusOK {
		if resp.StatusCode == http.StatusForbidden {
			log.Warn("403 waiting...")
			<-timeout
		}
		return nil, fmt.Errorf("Error status code: %d ", resp.StatusCode)
	}

	// 通过读取返回的1024个字节，来判断编码
	bufioReader := bufio.NewReader(resp.Body)
	charsetEncoding := determineEncoding(bufioReader)
	// 如果是GBK编码，则转换成utf8
	utf8Reader := transform.NewReader(bufioReader, charsetEncoding.NewDecoder())
	return ioutil.ReadAll(utf8Reader)
}

func determineEncoding(reader *bufio.Reader) encoding.Encoding {
	bytes, err := reader.Peek(1024)
	if err != nil {
		log.Printf("determine encoding error %v , using default utf8 ", err.Error())
		return unicode.UTF8
	}
	e, _, _ := charset.DetermineEncoding(bytes, "")
	return e
}
